{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Filters on the original amazon reviews dataset:\n",
    "1. coreset (reviewers who reviewed at least five things and products with at least five reviews)\n",
    "2. helpful (reviews with more helpful upvotes than unhelpful upvotes - requires at least one upvote)\n",
    "3. sentiment non-ambiguity (has to be rated 1, 3, or 5 -- no way to verify that a 2 is really a 2 ya know? its either positive middle or negative, but what really is a 4? so i drop all 2s and 4s)\n",
    "4. non-empty\n",
    "\n",
    "This results in ~ 10 million reviews."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# These imports enhance Python2/3 compatibility.\n",
    "from __future__ import print_function, absolute_import, division, unicode_literals, with_statement"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "from cleanlab.models.fasttext import FastTextClassifier, data_loader\n",
    "import cleanlab\n",
    "import numpy as np\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.model_selection import ParameterGrid\n",
    "import os\n",
    "from datetime import datetime as dt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_dir = \"/media/ssd/datasets/datasets/amazon5core/\"\n",
    "data_fn = \"amazon5core.json\"\n",
    "write_fn = 'amazon5core.txt'\n",
    "write_dir = \"/home/curtis/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Fetch and preprocess data\n",
    "\n",
    "need_text_data = False\n",
    "need_to_prepare_data_for_fasttext = False\n",
    "if need_to_prepare_data_for_fasttext:\n",
    "    # Convert amazon dataset to fasttext format\n",
    "    # Only include reviews with more helpful votes than unhelpful votes\n",
    "    # This takes about 6 minutes.\n",
    "    labels = []\n",
    "    with open(data_dir + data_fn, 'r') as rf:\n",
    "        with open(write_dir + write_fn, 'w') as wf:\n",
    "#             for i in range(1000000):\n",
    "#                 d = json.loads(rf.readline())\n",
    "            for line in rf:\n",
    "                d = json.loads(line)\n",
    "                h = d['helpful']\n",
    "                if h[0] > h[1] // 2:\n",
    "                    label = int(d['overall'])\n",
    "                    if label in [1,3,5]:\n",
    "                        text = d['reviewText']\n",
    "                        if len(text) > 0:\n",
    "                            wf.write(\"__label__{} {}\\n\".format(\n",
    "                                label, \n",
    "                                text.strip().replace('\\n', ' __newline__ '),\n",
    "                            ))\n",
    "                            labels.append(label)                          \n",
    "    label_map = {1:0, 3:1, 5:2}\n",
    "    labels = [label_map[l] for l in labels]\n",
    "else:\n",
    "    labels = np.empty(9996437, dtype=int)\n",
    "    if need_text_data:\n",
    "        text = []\n",
    "    loc = write_dir + 'amazon5core.preprocessed.txt'\n",
    "    bs = 1000000\n",
    "    label_map = {'__label__1':0, '__label__3':1, '__label__5':2}\n",
    "    for i, (l, t) in enumerate(data_loader(loc, batch_size=bs)):\n",
    "        labels[bs*i:bs*(i+1)] = [label_map[lab] for lab in l]\n",
    "        if need_text_data:\n",
    "            text.append(t)\n",
    "    if need_text_data:\n",
    "        text = [t for lst in text for t in lst]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "param_list = ParameterGrid({\n",
    "    \"cv_n_folds\" : [3],\n",
    "    \"lr\" : [.01, .05, 0.1, 0.5, 1.0],\n",
    "    \"ngram\" : [3],\n",
    "    \"epochs\" : [1, 5, 10],\n",
    "    \"dim\" : [100],\n",
    "})\n",
    "seed = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'cv_n_folds': 3, 'dim': 100, 'epochs': 1, 'lr': 0.01, 'ngram': 3}\n",
      "_create_train_data creating fastTextClf_1546460953.txt\n",
      "_create_train_data creating fastTextClf_1546461499.txt\n",
      "_create_train_data creating fastTextClf_1546462045.txt\n",
      "pyx finished. Writing: /home/curtis/amazon_pyx_cv__folds_3__epochs_1__lr_0.01__ngram_3__dim_100.npy\n",
      "Acc: 0.8803\n",
      "{'cv_n_folds': 3, 'dim': 100, 'epochs': 1, 'lr': 0.05, 'ngram': 3}\n",
      "Elapsed: 0:27:25 | Remaining: 6:23:57\n",
      "_create_train_data creating fastTextClf_1546462598.txt\n",
      "_create_train_data creating fastTextClf_1546463208.txt\n",
      "_create_train_data creating fastTextClf_1546463852.txt\n",
      "pyx finished. Writing: /home/curtis/amazon_pyx_cv__folds_3__epochs_1__lr_0.05__ngram_3__dim_100.npy\n",
      "Acc: 0.9039\n",
      "{'cv_n_folds': 3, 'dim': 100, 'epochs': 1, 'lr': 0.1, 'ngram': 3}\n",
      "Elapsed: 0:57:32 | Remaining: 6:14:04\n",
      "_create_train_data creating fastTextClf_1546464405.txt\n",
      "_create_train_data creating fastTextClf_1546465147.txt\n",
      "_create_train_data creating fastTextClf_1546466297.txt\n",
      "pyx finished. Writing: /home/curtis/amazon_pyx_cv__folds_3__epochs_1__lr_0.1__ngram_3__dim_100.npy\n",
      "Acc: 0.9054\n",
      "{'cv_n_folds': 3, 'dim': 100, 'epochs': 1, 'lr': 0.5, 'ngram': 3}\n",
      "Elapsed: 1:46:03 | Remaining: 7:04:15\n",
      "_create_train_data creating fastTextClf_1546467346.txt\n",
      "_create_train_data creating fastTextClf_1546468415.txt\n",
      "_create_train_data creating fastTextClf_1546469340.txt\n",
      "pyx finished. Writing: /home/curtis/amazon_pyx_cv__folds_3__epochs_1__lr_0.5__ngram_3__dim_100.npy\n",
      "Acc: 0.9056\n",
      "{'cv_n_folds': 3, 'dim': 100, 'epochs': 1, 'lr': 1.0, 'ngram': 3}\n",
      "Elapsed: 2:34:21 | Remaining: 7:04:28\n",
      "_create_train_data creating fastTextClf_1546470236.txt\n",
      "_create_train_data creating fastTextClf_1546471287.txt\n",
      "_create_train_data creating fastTextClf_1546472382.txt\n",
      "pyx finished. Writing: /home/curtis/amazon_pyx_cv__folds_3__epochs_1__lr_1.0__ngram_3__dim_100.npy\n",
      "Acc: 0.9061\n",
      "{'cv_n_folds': 3, 'dim': 100, 'epochs': 5, 'lr': 0.01, 'ngram': 3}\n",
      "Elapsed: 3:29:37 | Remaining: 6:59:15\n",
      "_create_train_data creating fastTextClf_1546473554.txt\n",
      "_create_train_data creating fastTextClf_1546475275.txt\n",
      "_create_train_data creating fastTextClf_1546476700.txt\n",
      "pyx finished. Writing: /home/curtis/amazon_pyx_cv__folds_3__epochs_5__lr_0.01__ngram_3__dim_100.npy\n",
      "Acc: 0.9062\n",
      "{'cv_n_folds': 3, 'dim': 100, 'epochs': 5, 'lr': 0.05, 'ngram': 3}\n",
      "Elapsed: 4:49:25 | Remaining: 7:14:08\n",
      "_create_train_data creating fastTextClf_1546478355.txt\n",
      "_create_train_data creating fastTextClf_1546480128.txt\n",
      "_create_train_data creating fastTextClf_1546482116.txt\n",
      "pyx finished. Writing: /home/curtis/amazon_pyx_cv__folds_3__epochs_5__lr_0.05__ngram_3__dim_100.npy\n",
      "Acc: 0.9081\n",
      "{'cv_n_folds': 3, 'dim': 100, 'epochs': 5, 'lr': 0.1, 'ngram': 3}\n",
      "Elapsed: 6:25:27 | Remaining: 7:20:31\n",
      "_create_train_data creating fastTextClf_1546484114.txt\n",
      "_create_train_data creating fastTextClf_1546486167.txt\n",
      "_create_train_data creating fastTextClf_1546488281.txt\n"
     ]
    }
   ],
   "source": [
    "# Fasttext model selection.\n",
    "\n",
    "start_time = dt.now()\n",
    "scores = []\n",
    "for i, params in enumerate(param_list):\n",
    "    print(params)    \n",
    "    if i > 0:\n",
    "        elapsed = dt.now() - start_time\n",
    "        total_time = elapsed * len(param_list) / float(i)\n",
    "        remaining = total_time - elapsed\n",
    "        print('Elapsed:', str(elapsed)[:-7], '| Remaining:', str(remaining)[:-7])\n",
    "    ftc = FastTextClassifier(\n",
    "        train_data_fn=write_dir + 'amazon5core.preprocessed.txt', \n",
    "        batch_size = 100000, \n",
    "        labels = [1, 3, 5],\n",
    "        kwargs_train_supervised = {\n",
    "            'epoch': params['epochs'],\n",
    "            'thread': 12,\n",
    "            'lr': params['lr'],\n",
    "            'wordNgrams': params['ngram'],\n",
    "            'bucket': 200000,\n",
    "            'dim': params['dim'],\n",
    "            'loss': 'softmax', #'softmax', # 'hs'\n",
    "        },\n",
    "    )\n",
    "    pyx = cleanlab.latent_estimation.estimate_cv_predicted_probabilities(\n",
    "        X=np.arange(len(labels)),\n",
    "        labels=labels,\n",
    "        clf=ftc,\n",
    "        cv_n_folds=params['cv_n_folds'],\n",
    "        seed=seed,\n",
    "    )\n",
    "    # Write out\n",
    "    wfn = write_dir + 'amazon_pyx_cv__folds_{}__epochs_{}__lr_{}__ngram_{}__dim_{}.npy'.format(\n",
    "        params['cv_n_folds'], params['epochs'], params['lr'], params['ngram'], params['dim'])\n",
    "    with open(wfn, 'wb') as wf:\n",
    "        np.save(wf, pyx)\n",
    "\n",
    "    # Check that probabilities are good.\n",
    "    print(\"pyx finished. Writing:\", wfn)\n",
    "    scores.append(accuracy_score(labels, np.argmax(pyx, axis = 1)))\n",
    "    print('Acc:', np.round(scores[-1], 4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best params {'ngram': 3, 'lr': 0.01, 'epochs': 10, 'dim': 100, 'cv_n_folds': 3}\n",
      "located in: /home/curtis/amazon_pyx_cv__folds_3__epochs_10__lr_1.0__ngram_3__dim_100.npy\n"
     ]
    }
   ],
   "source": [
    "best_params = param_list[np.argmax(scores)]\n",
    "print('best params', best_params)\n",
    "wfn = write_dir + 'amazon_pyx_cv__folds_{}__epochs_{}__lr_{}__ngram_{}__dim_{}.npy'.format(\n",
    "        params['cv_n_folds'], params['epochs'], params['lr'], params['ngram'], params['dim'])\n",
    "print('located in:', wfn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_from_scratch = False\n",
    "# Train the best model from scratch\n",
    "# No need to do this if you've already run the\n",
    "# hyper-parameter optimization above.\n",
    "if train_from_scratch:\n",
    "    cv_n_folds = 10 # Increasing more improves pyx, at great cost.\n",
    "    seed = 0\n",
    "    lr = .01\n",
    "    ngram = 3\n",
    "    epochs = 10 # Increasing more doesn't do much.\n",
    "    dim = 100\n",
    "\n",
    "    ftc = FastTextClassifier(\n",
    "        train_data_fn=write_dir + 'amazon5core.preprocessed.txt', \n",
    "        batch_size = 100000, \n",
    "        labels = [1, 3, 5],\n",
    "        kwargs_train_supervised = {\n",
    "            'epoch' : epochs,\n",
    "            'thread' : 12,\n",
    "            'lr' : lr,\n",
    "            'wordNgrams' : ngram,\n",
    "            'bucket' : 200000,\n",
    "            'dim' : dim,\n",
    "            'loss' : 'softmax', #'softmax', # 'hs'\n",
    "        }\n",
    "    )\n",
    "\n",
    "    pyx = cleanlab.latent_estimation.estimate_cv_predicted_probabilities(\n",
    "        X=np.arange(len(labels)),\n",
    "        labels=labels,\n",
    "        clf=ftc,\n",
    "        cv_n_folds=cv_n_folds,\n",
    "        seed=seed,\n",
    "    )\n",
    "\n",
    "    # Write out pyx\n",
    "    wfn = write_dir + 'amazon_pyx_cv__folds_{}__epochs_{}__lr_{}__ngram_{}__dim_{}.npy'.format(\n",
    "        cv_n_folds, epochs, lr, ngram, dim)\n",
    "    with open(wfn, 'wb') as wf:\n",
    "        np.save(wf, pyx)\n",
    "\n",
    "    # Check that probabilities are good.\n",
    "    print(wfn)\n",
    "    accuracy_score(labels, np.argmax(pyx, axis = 1))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
